# -*- coding: utf-8 -*-

# Scrapy settings for jingdong2 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'jingdong2'

SPIDER_MODULES = ['jingdong2.spiders']
NEWSPIDER_MODULE = 'jingdong2.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'jingdong2 (+http://www.yourdomain.com)'

# Obey robots.txt rules
# ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
    'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36',
    'cookie': 'unpl=V2_ZzNtbUZRQUV0WBZQKxxUAGIAFFURVUFHfFsRVn5MWVYzVBBfclRCFX0UR1FnGVsUZgoZWENcQRFFCEdkexhdBGYCEFpBU3NMJVZGV3lFFzVXABJtQlZzFXwPTld5GF4MbzMiWnJncxRFCXZVS1IyBCoDG1pKVEEUdwFOZHopXw%3d%3d; CCC_SE=ADC_TROFcwb2Hf86jnpWQjGg84wR3%2fMnXQY7sPWtzeWvdWj%2fJ9nLgs9W%2fx%2fj95Md%2ffNqlbS%2bl%2bZx2rp3NMo%2fneWiSbMnYp0ygSjp%2fXW9GN9TglDEpcJsed1jGlxviIvBqpCxI1hKGe8HPKZnX95KYlzuqlKKmyzfxIdYiUNrOU%2fV6gNW5%2b2jHhYVxZEOZC7kGHwVlDla8ke4EfTGPJINB6nkCjVJw2w1hKMT0Fj08KUiXzTz%2bqscigMqSYK2K%2fR4vws8Yngh2utfuewykytw1ltG1CwChXeJbJTGJHsPEcEtre5LgXl1wwK98Uybb%2bqZozIPT%2fzqGVoe0wdhKxnwx5JWh%2baKnin5cHRDlJsjMCeJMc2qzh2qNjO05U44Vn1S9pcyfWU6jJMR6Tj3UQtqEUJpNU301Y7I6ggMSNlpYCjC57r3gV%2fHAwISSOJ8Tt4QUvc6; __jda=122270672.1869230389.1555375235.1555375235.1555375237.1; __jdc=122270672; __jdv=122270672|site.baidu.com|t_1000003625_hao123mz|tuiguang|573a0aa5a4944279b33c8bf34d4bef33|1555375236617; __jdu=1869230389; areaId=1; xtest=3629.cf6b6759; ipLoc-djd=1-72-2799-0; shshshfp=71786fcde81bde2a06b5ba5d6259aaa4; shshshfpa=85c81148-985d-7992-735b-6f759c0eb7e0-1555375248; shshshfpb=iYMYfgpvPzuxj%2FNYPp49X%2Bg%3D%3D; rkv=V0400; 3AB9D23F7A4B3C9B=SEVCL5GYO4CGTCGWGNIVKBA24PBJO6BVUA4F7VOLF2HWID5XER3D2MLNSTEMDX2E7KJ2XKVR6ZSAUGFBIGN5QYLLEA; __jdb=122270672.4.1869230389|1.1555375237; shshshsID=b14eee8ea20b4cf62d69a9c687d7a7c8_3_1555375315449; qrsc=3',
    'referer': 'https://search.jd.com/Search?keyword=%E7%BE%8E%E9%A3%9F&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E7%BE%8E%E9%A3%9F&stock=1&page=1&s=1&click=0'
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = {
#    'jingdong2.middlewares.Jingdong2SpiderMiddleware': 543,
# }

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    'jingdong2.middlewares.Jingdong2DownloaderMiddleware': 543,
# }

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
# }

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    'jingdong2.pipelines.Jingdong2Pipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True
# The initial download delay
# AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
# AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
# AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
# AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
